package com.zhangyx.crawler.core;

import com.zhangyx.crawler.comm.RedisUtil;
import org.apache.log4j.Logger;

public abstract class Crawler extends Thread{

    private static Logger log = Logger.getLogger(Crawler.class);
    /**
     * 抓取间隔时间-毫秒
     * @return
     */
    public abstract int intervalTime();

    /**
     * 下载网页
     * @return
     */
    public abstract Page download();

    /**
     * 处理网页--存缓存、验证网页是否正常
     * @param page
     */
    public abstract void deal(Page page);

    public void sleepMills(int mills) {
        try {
            sleep(mills);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    protected void savePage(String key, Page page) {
        RedisUtil.pushStackObj(key, page);
    }

    public Page getNextPage(String key) {
        log.info("缓存的长度-->" + RedisUtil.length(key));
        Page page = RedisUtil.popStackObj(key, Page.class);
        while (page == null) {
            log.info("缓存" + key + "中没有数据了，休息1秒");
            sleepMills(1000);
            page = RedisUtil.popStackObj(key, Page.class);
        }
        return page;
    }

    @Override
    public void run() {
        while (!interrupted()) {
            deal(download());
            try {
                sleep(intervalTime());
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        System.exit(0);
    }

}
